{
  "cells": [
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# `tf_adjustment_chart`\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "tags": [
          "hide_input"
        ]
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "<style>\n",
              "  #altair-viz-892313d0af8c4befa69ed9691ea6b197.vega-embed {\n",
              "    width: 100%;\n",
              "    display: flex;\n",
              "  }\n",
              "\n",
              "  #altair-viz-892313d0af8c4befa69ed9691ea6b197.vega-embed details,\n",
              "  #altair-viz-892313d0af8c4befa69ed9691ea6b197.vega-embed details summary {\n",
              "    position: relative;\n",
              "  }\n",
              "</style>\n",
              "<div id=\"altair-viz-892313d0af8c4befa69ed9691ea6b197\"></div>\n",
              "<script type=\"text/javascript\">\n",
              "  var VEGA_DEBUG = (typeof VEGA_DEBUG == \"undefined\") ? {} : VEGA_DEBUG;\n",
              "  (function(spec, embedOpt){\n",
              "    let outputDiv = document.currentScript.previousElementSibling;\n",
              "    if (outputDiv.id !== \"altair-viz-892313d0af8c4befa69ed9691ea6b197\") {\n",
              "      outputDiv = document.getElementById(\"altair-viz-892313d0af8c4befa69ed9691ea6b197\");\n",
              "    }\n",
              "    const paths = {\n",
              "      \"vega\": \"https://cdn.jsdelivr.net/npm/vega@5?noext\",\n",
              "      \"vega-lib\": \"https://cdn.jsdelivr.net/npm/vega-lib?noext\",\n",
              "      \"vega-lite\": \"https://cdn.jsdelivr.net/npm/vega-lite@5.17.0?noext\",\n",
              "      \"vega-embed\": \"https://cdn.jsdelivr.net/npm/vega-embed@6?noext\",\n",
              "    };\n",
              "\n",
              "    function maybeLoadScript(lib, version) {\n",
              "      var key = `${lib.replace(\"-\", \"\")}_version`;\n",
              "      return (VEGA_DEBUG[key] == version) ?\n",
              "        Promise.resolve(paths[lib]) :\n",
              "        new Promise(function(resolve, reject) {\n",
              "          var s = document.createElement('script');\n",
              "          document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
              "          s.async = true;\n",
              "          s.onload = () => {\n",
              "            VEGA_DEBUG[key] = version;\n",
              "            return resolve(paths[lib]);\n",
              "          };\n",
              "          s.onerror = () => reject(`Error loading script: ${paths[lib]}`);\n",
              "          s.src = paths[lib];\n",
              "        });\n",
              "    }\n",
              "\n",
              "    function showError(err) {\n",
              "      outputDiv.innerHTML = `<div class=\"error\" style=\"color:red;\">${err}</div>`;\n",
              "      throw err;\n",
              "    }\n",
              "\n",
              "    function displayChart(vegaEmbed) {\n",
              "      vegaEmbed(outputDiv, spec, embedOpt)\n",
              "        .catch(err => showError(`Javascript Error: ${err.message}<br>This usually means there's a typo in your chart specification. See the javascript console for the full traceback.`));\n",
              "    }\n",
              "\n",
              "    if(typeof define === \"function\" && define.amd) {\n",
              "      requirejs.config({paths});\n",
              "      require([\"vega-embed\"], displayChart, err => showError(`Error loading script: ${err.message}`));\n",
              "    } else {\n",
              "      maybeLoadScript(\"vega\", \"5\")\n",
              "        .then(() => maybeLoadScript(\"vega-lite\", \"5.17.0\"))\n",
              "        .then(() => maybeLoadScript(\"vega-embed\", \"6\"))\n",
              "        .catch(showError)\n",
              "        .then(() => displayChart(vegaEmbed));\n",
              "    }\n",
              "  })({\"config\": {\"view\": {\"continuousWidth\": 300, \"continuousHeight\": 300}, \"params\": [{\"name\": \"gamma_sel\", \"bind\": {\"input\": \"select\", \"options\": [3], \"labels\": [\"Exact match on first_name (TF col: first_name)\"], \"name\": \"Gamma level:\"}, \"value\": 3}]}, \"hconcat\": [{\"layer\": [{\"mark\": {\"type\": \"point\", \"filled\": true, \"size\": 100, \"stroke\": \"black\", \"strokeWidth\": 1}, \"encoding\": {\"color\": {\"field\": \"log2_bf_tf\", \"scale\": {\"domain\": [-2.5, 2.5], \"scheme\": \"redyellowgreen\"}, \"title\": \"TF adjustment weight\", \"type\": \"quantitative\"}, \"tooltip\": [{\"field\": \"value\", \"title\": \"Value\", \"type\": \"nominal\"}, {\"field\": \"log2_bf\", \"format\": \"+.3\", \"title\": \"Match weight\", \"type\": \"quantitative\"}, {\"field\": \"log2_bf_tf\", \"format\": \"+.3\", \"title\": \"TF adjusted match weight\", \"type\": \"quantitative\"}, {\"field\": \"log2_bf_final\", \"format\": \"+.3\", \"title\": \"Final match weight\", \"type\": \"quantitative\"}], \"x\": {\"axis\": {\"labelAngle\": -60, \"labelFontSize\": 16, \"titleFontSize\": 20}, \"field\": \"value\", \"sort\": {\"field\": \"log2_bf_final\", \"order\": \"ascending\"}, \"title\": \"TF column value\", \"type\": \"nominal\"}, \"y\": {\"axis\": {\"format\": \"+\", \"labelFontSize\": 16, \"titleFontSize\": 18, \"values\": [-5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]}, \"field\": \"log2_bf_final\", \"title\": \"Match weight\", \"type\": \"quantitative\"}}}, {\"mark\": \"rule\", \"encoding\": {\"y\": {\"field\": \"log2_bf\", \"type\": \"quantitative\"}}, \"transform\": [{\"filter\": \"datum.gamma == gamma_sel\"}]}, {\"mark\": {\"type\": \"rule\", \"opacity\": 0.5, \"strokeWidth\": 2}, \"encoding\": {\"color\": {\"field\": \"log2_bf_tf\", \"legend\": null, \"scale\": {\"domain\": [-2.5, 2.5], \"scheme\": \"redyellowgreen\"}, \"title\": \"TF adjustment weight\", \"type\": \"quantitative\"}, \"x\": {\"field\": \"value\", \"sort\": {\"field\": \"log2_bf_final\", \"order\": \"ascending\"}, \"title\": \"TF column value\", \"type\": \"nominal\"}, \"y\": {\"field\": \"log2_bf_final\", \"type\": \"quantitative\"}, \"y2\": {\"type\": \"quantitative\"}}, \"transform\": [{\"filter\": \"datum.gamma == gamma_sel\"}]}], \"data\": {\"name\": \"data\"}, \"height\": 400, \"transform\": [{\"filter\": \"datum.gamma == gamma_sel\"}], \"width\": {\"step\": 20}}, {\"mark\": {\"type\": \"bar\", \"fillOpacity\": 0.8, \"filled\": true, \"stroke\": \"black\", \"strokeWidth\": 1}, \"data\": {\"name\": \"hist\"}, \"encoding\": {\"color\": {\"field\": \"log2_bf_tf\", \"legend\": null, \"scale\": {\"domain\": [-2.5, 2.5], \"scheme\": \"redyellowgreen\"}, \"title\": \"TF adjustment weight\", \"type\": \"quantitative\"}, \"tooltip\": [{\"field\": \"log2_bf_desc\", \"title\": \"Match weight\"}, {\"field\": \"count\", \"title\": \"Number of values\"}], \"x\": {\"axis\": {\"domain\": false, \"gridOpacity\": 0.5, \"labelAlign\": \"center\", \"labelFontSize\": 12, \"labelOpacity\": 0.5, \"labelOverlap\": true, \"ticks\": false, \"title\": \"Count of values\", \"titleFontSize\": 12, \"titleOpacity\": 0.5}, \"field\": \"count\", \"type\": \"quantitative\"}, \"y\": {\"axis\": null, \"bin\": {\"step\": 0.5}, \"field\": \"log2_bf_final\", \"type\": \"quantitative\"}}, \"transform\": [{\"filter\": {\"or\": [\"datum.gamma == gamma_sel\", {\"field\": \"gamma\", \"valid\": false}]}}], \"view\": {\"stroke\": \"transparent\"}, \"width\": 100}], \"datasets\": {\"data\": [{\"value\": \"Oscra\", \"tf\": 0.0012033694344163659, \"label_for_charts\": \"Exact match on first_name\", \"u_probability\": 0.0057935713975033705, \"tf_col\": \"first_name\", \"tf_adjustment_weight\": 1.0, \"log2_bf\": 6.40519430138448, \"gamma\": 3, \"log2_bf_tf\": 2.267373341558131, \"log2_bf_final\": 8.67256764294261, \"most_freq_rank\": 334, \"least_freq_rank\": 0}, {\"value\": \"foSia\", \"tf\": 0.0012033694344163659, \"label_for_charts\": \"Exact match on first_name\", \"u_probability\": 0.0057935713975033705, \"tf_col\": \"first_name\", \"tf_adjustment_weight\": 1.0, \"log2_bf\": 6.40519430138448, \"gamma\": 3, \"log2_bf_tf\": 2.267373341558131, \"log2_bf_final\": 8.67256764294261, \"most_freq_rank\": 333, \"least_freq_rank\": 1}, {\"value\": \"hNoaN\", \"tf\": 0.0012033694344163659, \"label_for_charts\": \"Exact match on first_name\", \"u_probability\": 0.0057935713975033705, \"tf_col\": \"first_name\", \"tf_adjustment_weight\": 1.0, \"log2_bf\": 6.40519430138448, \"gamma\": 3, \"log2_bf_tf\": 2.267373341558131, \"log2_bf_final\": 8.67256764294261, \"most_freq_rank\": 332, \"least_freq_rank\": 2}, {\"value\": \"EvEa\", \"tf\": 0.0012033694344163659, \"label_for_charts\": \"Exact match on first_name\", \"u_probability\": 0.0057935713975033705, \"tf_col\": \"first_name\", \"tf_adjustment_weight\": 1.0, \"log2_bf\": 6.40519430138448, \"gamma\": 3, \"log2_bf_tf\": 2.267373341558131, \"log2_bf_final\": 8.67256764294261, \"most_freq_rank\": 331, \"least_freq_rank\": 3}, {\"value\": \"Imgeen\", \"tf\": 0.0012033694344163659, \"label_for_charts\": \"Exact match on first_name\", \"u_probability\": 0.0057935713975033705, \"tf_col\": \"first_name\", \"tf_adjustment_weight\": 1.0, \"log2_bf\": 6.40519430138448, \"gamma\": 3, \"log2_bf_tf\": 2.267373341558131, \"log2_bf_final\": 8.67256764294261, \"most_freq_rank\": 330, \"least_freq_rank\": 4}, {\"value\": \"geeorGe\", \"tf\": 0.0012033694344163659, \"label_for_charts\": \"Exact match on first_name\", \"u_probability\": 0.0057935713975033705, \"tf_col\": \"first_name\", \"tf_adjustment_weight\": 1.0, \"log2_bf\": 6.40519430138448, \"gamma\": 3, \"log2_bf_tf\": 2.267373341558131, \"log2_bf_final\": 8.67256764294261, \"most_freq_rank\": 329, \"least_freq_rank\": 5}, {\"value\": \"Lydiia\", \"tf\": 0.0012033694344163659, \"label_for_charts\": \"Exact match on first_name\", \"u_probability\": 0.0057935713975033705, \"tf_col\": \"first_name\", \"tf_adjustment_weight\": 1.0, \"log2_bf\": 6.40519430138448, \"gamma\": 3, \"log2_bf_tf\": 2.267373341558131, \"log2_bf_final\": 8.67256764294261, \"most_freq_rank\": 328, \"least_freq_rank\": 6}, {\"value\": \"RoRyn\", \"tf\": 0.0012033694344163659, \"label_for_charts\": \"Exact match on first_name\", \"u_probability\": 0.0057935713975033705, \"tf_col\": \"first_name\", \"tf_adjustment_weight\": 1.0, \"log2_bf\": 6.40519430138448, \"gamma\": 3, \"log2_bf_tf\": 2.267373341558131, \"log2_bf_final\": 8.67256764294261, \"most_freq_rank\": 327, \"least_freq_rank\": 7}, {\"value\": \"Olirev\", \"tf\": 0.0012033694344163659, \"label_for_charts\": \"Exact match on first_name\", \"u_probability\": 0.0057935713975033705, \"tf_col\": \"first_name\", \"tf_adjustment_weight\": 1.0, \"log2_bf\": 6.40519430138448, \"gamma\": 3, \"log2_bf_tf\": 2.267373341558131, \"log2_bf_final\": 8.67256764294261, \"most_freq_rank\": 326, \"least_freq_rank\": 8}, {\"value\": \"Teheo\", \"tf\": 0.0012033694344163659, \"label_for_charts\": \"Exact match on first_name\", \"u_probability\": 0.0057935713975033705, \"tf_col\": \"first_name\", \"tf_adjustment_weight\": 1.0, \"log2_bf\": 6.40519430138448, \"gamma\": 3, \"log2_bf_tf\": 2.267373341558131, \"log2_bf_final\": 8.67256764294261, \"most_freq_rank\": 325, \"least_freq_rank\": 9}, {\"value\": \"Robert\", \"tf\": 0.0036101083032490976, \"label_for_charts\": \"Exact match on first_name\", \"u_probability\": 0.0057935713975033705, \"tf_col\": \"first_name\", \"tf_adjustment_weight\": 1.0, \"log2_bf\": 6.40519430138448, \"gamma\": 3, \"log2_bf_tf\": 0.6824108408369749, \"log2_bf_final\": 7.087605142221455, \"most_freq_rank\": 84, \"least_freq_rank\": 250}, {\"value\": \"Grace\", \"tf\": 0.006016847172081829, \"label_for_charts\": \"Exact match on first_name\", \"u_probability\": 0.0057935713975033705, \"tf_col\": \"first_name\", \"tf_adjustment_weight\": 1.0, \"log2_bf\": 6.40519430138448, \"gamma\": 3, \"log2_bf_tf\": -0.05455475332923131, \"log2_bf_final\": 6.350639548055248, \"most_freq_rank\": 42, \"least_freq_rank\": 292}, {\"value\": \"Theodore\", \"tf\": 0.012033694344163659, \"label_for_charts\": \"Exact match on first_name\", \"u_probability\": 0.0057935713975033705, \"tf_col\": \"first_name\", \"tf_adjustment_weight\": 1.0, \"log2_bf\": 6.40519430138448, \"gamma\": 3, \"log2_bf_tf\": -1.0545547533292312, \"log2_bf_final\": 5.350639548055248, \"most_freq_rank\": 9, \"least_freq_rank\": 325}, {\"value\": \"Elizabeth\", \"tf\": 0.013237063778580024, \"label_for_charts\": \"Exact match on first_name\", \"u_probability\": 0.0057935713975033705, \"tf_col\": \"first_name\", \"tf_adjustment_weight\": 1.0, \"log2_bf\": 6.40519430138448, \"gamma\": 3, \"log2_bf_tf\": -1.1920582770791661, \"log2_bf_final\": 5.213136024305314, \"most_freq_rank\": 8, \"least_freq_rank\": 326}, {\"value\": \"Alfie\", \"tf\": 0.013237063778580024, \"label_for_charts\": \"Exact match on first_name\", \"u_probability\": 0.0057935713975033705, \"tf_col\": \"first_name\", \"tf_adjustment_weight\": 1.0, \"log2_bf\": 6.40519430138448, \"gamma\": 3, \"log2_bf_tf\": -1.1920582770791661, \"log2_bf_final\": 5.213136024305314, \"most_freq_rank\": 7, \"least_freq_rank\": 327}, {\"value\": \"Jessica\", \"tf\": 0.013237063778580024, \"label_for_charts\": \"Exact match on first_name\", \"u_probability\": 0.0057935713975033705, \"tf_col\": \"first_name\", \"tf_adjustment_weight\": 1.0, \"log2_bf\": 6.40519430138448, \"gamma\": 3, \"log2_bf_tf\": -1.1920582770791661, \"log2_bf_final\": 5.213136024305314, \"most_freq_rank\": 6, \"least_freq_rank\": 328}, {\"value\": \"George\", \"tf\": 0.01444043321299639, \"label_for_charts\": \"Exact match on first_name\", \"u_probability\": 0.0057935713975033705, \"tf_col\": \"first_name\", \"tf_adjustment_weight\": 1.0, \"log2_bf\": 6.40519430138448, \"gamma\": 3, \"log2_bf_tf\": -1.317589159163025, \"log2_bf_final\": 5.087605142221455, \"most_freq_rank\": 5, \"least_freq_rank\": 329}, {\"value\": \"James\", \"tf\": 0.015643802647412757, \"label_for_charts\": \"Exact match on first_name\", \"u_probability\": 0.0057935713975033705, \"tf_col\": \"first_name\", \"tf_adjustment_weight\": 1.0, \"log2_bf\": 6.40519430138448, \"gamma\": 3, \"log2_bf_tf\": -1.4330663765829612, \"log2_bf_final\": 4.972127924801518, \"most_freq_rank\": 4, \"least_freq_rank\": 330}, {\"value\": \"Olivia\", \"tf\": 0.01684717208182912, \"label_for_charts\": \"Exact match on first_name\", \"u_probability\": 0.0057935713975033705, \"tf_col\": \"first_name\", \"tf_adjustment_weight\": 1.0, \"log2_bf\": 6.40519430138448, \"gamma\": 3, \"log2_bf_tf\": -1.539981580499473, \"log2_bf_final\": 4.8652127208850064, \"most_freq_rank\": 3, \"least_freq_rank\": 331}, {\"value\": \"Freddie\", \"tf\": 0.018050541516245487, \"label_for_charts\": \"Exact match on first_name\", \"u_probability\": 0.0057935713975033705, \"tf_col\": \"first_name\", \"tf_adjustment_weight\": 1.0, \"log2_bf\": 6.40519430138448, \"gamma\": 3, \"log2_bf_tf\": -1.6395172540503875, \"log2_bf_final\": 4.765677047334092, \"most_freq_rank\": 2, \"least_freq_rank\": 332}, {\"value\": \"Jacob\", \"tf\": 0.019253910950661854, \"label_for_charts\": \"Exact match on first_name\", \"u_probability\": 0.0057935713975033705, \"tf_col\": \"first_name\", \"tf_adjustment_weight\": 1.0, \"log2_bf\": 6.40519430138448, \"gamma\": 3, \"log2_bf_tf\": -1.7326266584418688, \"log2_bf_final\": 4.672567642942611, \"most_freq_rank\": 1, \"least_freq_rank\": 333}, {\"value\": \"Oliver\", \"tf\": 0.03369434416365824, \"label_for_charts\": \"Exact match on first_name\", \"u_probability\": 0.0057935713975033705, \"tf_col\": \"first_name\", \"tf_adjustment_weight\": 1.0, \"log2_bf\": 6.40519430138448, \"gamma\": 3, \"log2_bf_tf\": -2.539981580499473, \"log2_bf_final\": 3.865212720885007, \"most_freq_rank\": 0, \"least_freq_rank\": 334}], \"hist\": [{\"gamma\": 3, \"log2_bf\": 6.40519430138448, \"count\": 1, \"log2_bf_tf\": -2.539981580499473, \"bin_start\": 3.5, \"bin_end\": 4.0, \"log2_bf_final\": 3.75, \"log2_bf_desc\": \"3.5-4.0\"}, {\"gamma\": 3, \"log2_bf\": 6.40519430138448, \"count\": 4, \"log2_bf_tf\": -1.5862979673936726, \"bin_start\": 4.5, \"bin_end\": 5.0, \"log2_bf_final\": 4.75, \"log2_bf_desc\": \"4.5-5.0\"}, {\"gamma\": 3, \"log2_bf\": 6.40519430138448, \"count\": 6, \"log2_bf_tf\": -1.1671455828431643, \"bin_start\": 5.0, \"bin_end\": 5.5, \"log2_bf_final\": 5.25, \"log2_bf_desc\": \"5.0-5.5\"}, {\"gamma\": 3, \"log2_bf\": 6.40519430138448, \"count\": 12, \"log2_bf_tf\": -0.739213716395367, \"bin_start\": 5.5, \"bin_end\": 6.0, \"log2_bf_final\": 5.75, \"log2_bf_desc\": \"5.5-6.0\"}, {\"gamma\": 3, \"log2_bf\": 6.40519430138448, \"count\": 25, \"log2_bf_tf\": -0.15976851566274886, \"bin_start\": 6.0, \"bin_end\": 6.5, \"log2_bf_final\": 6.25, \"log2_bf_desc\": \"6.0-6.5\"}, {\"gamma\": 3, \"log2_bf\": 6.40519430138448, \"count\": 24, \"log2_bf_tf\": 0.2673733415581312, \"bin_start\": 6.5, \"bin_end\": 7.0, \"log2_bf_final\": 6.75, \"log2_bf_desc\": \"6.5-7.0\"}, {\"gamma\": 3, \"log2_bf\": 6.40519430138448, \"count\": 24, \"log2_bf_tf\": 0.6824108408369748, \"bin_start\": 7.0, \"bin_end\": 7.5, \"log2_bf_final\": 7.25, \"log2_bf_desc\": \"7.0-7.5\"}, {\"gamma\": 3, \"log2_bf\": 6.40519430138448, \"count\": 41, \"log2_bf_tf\": 1.2673733415581312, \"bin_start\": 7.5, \"bin_end\": 8.0, \"log2_bf_final\": 7.75, \"log2_bf_desc\": \"7.5-8.0\"}, {\"gamma\": 3, \"log2_bf\": 6.40519430138448, \"count\": 198, \"log2_bf_tf\": 2.267373341558131, \"bin_start\": 8.5, \"bin_end\": 9.0, \"log2_bf_final\": 8.75, \"log2_bf_desc\": \"8.5-9.0\"}]}, \"resolve\": {\"scale\": {\"color\": \"shared\", \"y\": \"shared\"}}, \"spacing\": 10, \"title\": {\"text\": \"Term frequency adjusted match weights\", \"anchor\": \"middle\", \"fontSize\": 16, \"subtitle\": \"For selected values, incl. the lowest and highest frequency\"}, \"$schema\": \"https://vega.github.io/schema/vega-lite/v5.9.3.json\"}, {\"mode\": \"vega-lite\"});\n",
              "</script>"
            ],
            "text/plain": [
              "alt.HConcatChart(...)"
            ]
          },
          "execution_count": 3,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "chart"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "!!! info \"At a glance\"\n",
        "    **Useful for:** Looking at the impact of Term Frequency Adjustments on Match Weights.\n",
        "\n",
        "    **API Documentation:** [tf_adjustment_chart()](../api_docs/visualisations.md#splink.internals.linker_components.visualisations.LinkerVisualisations.tf_adjustment_chart)\n",
        "\n",
        "    **What is needed to generate the chart?:** A trained Splink model, including comparisons with term frequency adjustments.\n"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### What the chart shows\n",
        "\n",
        "The `tf_adjustment_chart` shows the impact of Term Frequency Adjustments on the Match Weight of a comparison. It is made up of two charts for each selected comparison:\n",
        "\n",
        "- The left chart shows the match weight for two records with a matching `first_name` including a term frequency adjustment. The black horizontal line represents the base match weight (i.e. with no term frequency adjustment applied). By default this chart contains the 10 most frequent and 10 least frequent values in a comparison as well as any values assigned in the `vals_to_include` parameter.\n",
        "- The right chart shows the distribution of match weights across all of the values of `first_name`.\n"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "??? note \"What the tooltip shows\"\n",
        "\n",
        "    #### Left chart\n",
        "\n",
        "    ![](./img/tf_adjustment_chart_tooltip_1.png)\n",
        "\n",
        "    The tooltip shows a number of statistics based on the column value of the point theat the user is hovering over, including:\n",
        "\n",
        "    - The column value\n",
        "    - The base match weight (i.e. with no term frequency adjustment) for a match on the column.\n",
        "    - The term frequency adjustment for the column value.\n",
        "    - The final match weight (i.e. the combined base match weight and term frequency adjustment)\n",
        "\n",
        "    #### Right chart\n",
        "\n",
        "    ![](./img/tf_adjustment_chart_tooltip_2.png)\n",
        "\n",
        "    The tooltip shows a number of statistics based on the bar that the user is hovering over, including:\n",
        "\n",
        "    - The final match weight bucket (in steps of 0.5).\n",
        "    - The number of records with a final match weight in the final match weight bucket.\n"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "<hr>\n"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### How to interpret the chart\n",
        "\n",
        "The most common terms (on the left of the first chart) will have a negative term frequency adjustment and the values on the chart and represent the lowest match weight for a match for the selected comparison. Conversely, the least common terms (on the right of the first chart) will have a positive term frequency adjustment and the values on the chart represent the highest match weight for a match for the selected comparison.\n",
        "\n",
        "Given that the first chart only shows the most and least frequently occuring values, the second chart is provided to show the distribution of final match weights (including term frequency adjustments) across all values in the dataset.\n"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "<hr>\n"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Actions to take as a result of the chart\n",
        "\n",
        "There are no direct actions that need to be taken as a result of this chart. It is intended to give the user an indication of the size of the impact of Term Frequency Adjustments on comparisons, as seen in the Waterfall Chart.\n"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Worked Example\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "tags": [
          "hide_output"
        ]
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "You are using the default value for `max_pairs`, which may be too small and thus lead to inaccurate estimates for your model's u-parameters. Consider increasing to 1e8 or 1e9, which will result in more accurate estimates, but with a longer run time.\n",
            "----- Estimating u probabilities using random sampling -----\n",
            "u probability not trained for dob - Abs difference of 'transformed dob <= 1 month' (comparison vector value: 1). This usually means the comparison level was never observed in the training data.\n",
            "\n",
            "Estimated u probabilities using random sampling\n",
            "\n",
            "Your model is not yet fully trained. Missing estimates for:\n",
            "    - first_name (no m values are trained).\n",
            "    - surname (no m values are trained).\n",
            "    - dob (some u values are not trained, no m values are trained).\n",
            "    - city (no m values are trained).\n",
            "    - email (no m values are trained).\n",
            "\n",
            "----- Starting EM training session -----\n",
            "\n",
            "Estimating the m probabilities of the model by blocking on:\n",
            "(l.\"first_name\" = r.\"first_name\") AND (l.\"surname\" = r.\"surname\")\n",
            "\n",
            "Parameter estimates will be made for the following comparison(s):\n",
            "    - dob\n",
            "    - city\n",
            "    - email\n",
            "\n",
            "Parameter estimates cannot be made for the following comparison(s) since they are used in the blocking rules: \n",
            "    - first_name\n",
            "    - surname\n",
            "\n",
            "WARNING:\n",
            "Level Abs difference of 'transformed dob <= 1 month' on comparison dob not observed in dataset, unable to train m value\n",
            "\n",
            "WARNING:\n",
            "Level Jaro-Winkler distance of transformed email >= 0.88 on comparison email not observed in dataset, unable to train m value\n",
            "\n",
            "Iteration 1: Largest change in params was -0.466 in the m_probability of dob, level `Exact match on dob`\n",
            "Iteration 2: Largest change in params was 0.141 in probability_two_random_records_match\n",
            "Iteration 3: Largest change in params was 0.0319 in probability_two_random_records_match\n",
            "Iteration 4: Largest change in params was 0.0105 in probability_two_random_records_match\n",
            "Iteration 5: Largest change in params was 0.00435 in probability_two_random_records_match\n",
            "Iteration 6: Largest change in params was 0.00208 in probability_two_random_records_match\n",
            "Iteration 7: Largest change in params was 0.00109 in probability_two_random_records_match\n",
            "Iteration 8: Largest change in params was 0.000601 in probability_two_random_records_match\n",
            "Iteration 9: Largest change in params was 0.000342 in probability_two_random_records_match\n",
            "Iteration 10: Largest change in params was 0.000197 in probability_two_random_records_match\n",
            "Iteration 11: Largest change in params was 0.000115 in probability_two_random_records_match\n",
            "Iteration 12: Largest change in params was 6.75e-05 in probability_two_random_records_match\n",
            "\n",
            "EM converged after 12 iterations\n",
            "m probability not trained for dob - Abs difference of 'transformed dob <= 1 month' (comparison vector value: 1). This usually means the comparison level was never observed in the training data.\n",
            "m probability not trained for email - Jaro-Winkler distance of transformed email >= 0.88 (comparison vector value: 1). This usually means the comparison level was never observed in the training data.\n",
            "\n",
            "Your model is not yet fully trained. Missing estimates for:\n",
            "    - first_name (no m values are trained).\n",
            "    - surname (no m values are trained).\n",
            "    - dob (some u values are not trained, some m values are not trained).\n",
            "    - email (some m values are not trained).\n",
            "\n",
            "----- Starting EM training session -----\n",
            "\n",
            "Estimating the m probabilities of the model by blocking on:\n",
            "l.\"dob\" = r.\"dob\"\n",
            "\n",
            "Parameter estimates will be made for the following comparison(s):\n",
            "    - first_name\n",
            "    - surname\n",
            "    - city\n",
            "    - email\n",
            "\n",
            "Parameter estimates cannot be made for the following comparison(s) since they are used in the blocking rules: \n",
            "    - dob\n",
            "\n",
            "WARNING:\n",
            "Level Jaro-Winkler distance of transformed email >= 0.88 on comparison email not observed in dataset, unable to train m value\n",
            "\n",
            "Iteration 1: Largest change in params was 0.638 in probability_two_random_records_match\n",
            "Iteration 2: Largest change in params was 0.178 in probability_two_random_records_match\n",
            "Iteration 3: Largest change in params was 0.0846 in the m_probability of first_name, level `All other comparisons`\n",
            "Iteration 4: Largest change in params was 0.0268 in probability_two_random_records_match\n",
            "Iteration 5: Largest change in params was 0.0101 in probability_two_random_records_match\n",
            "Iteration 6: Largest change in params was 0.00431 in probability_two_random_records_match\n",
            "Iteration 7: Largest change in params was 0.00198 in probability_two_random_records_match\n",
            "Iteration 8: Largest change in params was 0.000936 in probability_two_random_records_match\n",
            "Iteration 9: Largest change in params was 0.00045 in probability_two_random_records_match\n",
            "Iteration 10: Largest change in params was 0.000218 in probability_two_random_records_match\n",
            "Iteration 11: Largest change in params was 0.000106 in probability_two_random_records_match\n",
            "Iteration 12: Largest change in params was 5.19e-05 in probability_two_random_records_match\n",
            "\n",
            "EM converged after 12 iterations\n",
            "m probability not trained for email - Jaro-Winkler distance of transformed email >= 0.88 (comparison vector value: 1). This usually means the comparison level was never observed in the training data.\n",
            "\n",
            "Your model is not yet fully trained. Missing estimates for:\n",
            "    - dob (some u values are not trained, some m values are not trained).\n",
            "    - email (some m values are not trained).\n",
            "/Users/robinlinacre/Documents/data_linking/splink_4_311/splink/internals/linker_components/visualisations.py:192: UserWarning: Values ['Robert', 'Grace'] from `vals_to_include` were not found in the dataset so are not included in the chart.\n",
            "  return tf_adjustment_chart(\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "<style>\n",
              "  #altair-viz-056c9c44ffb84c8aa10628fa14f9020b.vega-embed {\n",
              "    width: 100%;\n",
              "    display: flex;\n",
              "  }\n",
              "\n",
              "  #altair-viz-056c9c44ffb84c8aa10628fa14f9020b.vega-embed details,\n",
              "  #altair-viz-056c9c44ffb84c8aa10628fa14f9020b.vega-embed details summary {\n",
              "    position: relative;\n",
              "  }\n",
              "</style>\n",
              "<div id=\"altair-viz-056c9c44ffb84c8aa10628fa14f9020b\"></div>\n",
              "<script type=\"text/javascript\">\n",
              "  var VEGA_DEBUG = (typeof VEGA_DEBUG == \"undefined\") ? {} : VEGA_DEBUG;\n",
              "  (function(spec, embedOpt){\n",
              "    let outputDiv = document.currentScript.previousElementSibling;\n",
              "    if (outputDiv.id !== \"altair-viz-056c9c44ffb84c8aa10628fa14f9020b\") {\n",
              "      outputDiv = document.getElementById(\"altair-viz-056c9c44ffb84c8aa10628fa14f9020b\");\n",
              "    }\n",
              "    const paths = {\n",
              "      \"vega\": \"https://cdn.jsdelivr.net/npm/vega@5?noext\",\n",
              "      \"vega-lib\": \"https://cdn.jsdelivr.net/npm/vega-lib?noext\",\n",
              "      \"vega-lite\": \"https://cdn.jsdelivr.net/npm/vega-lite@5.17.0?noext\",\n",
              "      \"vega-embed\": \"https://cdn.jsdelivr.net/npm/vega-embed@6?noext\",\n",
              "    };\n",
              "\n",
              "    function maybeLoadScript(lib, version) {\n",
              "      var key = `${lib.replace(\"-\", \"\")}_version`;\n",
              "      return (VEGA_DEBUG[key] == version) ?\n",
              "        Promise.resolve(paths[lib]) :\n",
              "        new Promise(function(resolve, reject) {\n",
              "          var s = document.createElement('script');\n",
              "          document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
              "          s.async = true;\n",
              "          s.onload = () => {\n",
              "            VEGA_DEBUG[key] = version;\n",
              "            return resolve(paths[lib]);\n",
              "          };\n",
              "          s.onerror = () => reject(`Error loading script: ${paths[lib]}`);\n",
              "          s.src = paths[lib];\n",
              "        });\n",
              "    }\n",
              "\n",
              "    function showError(err) {\n",
              "      outputDiv.innerHTML = `<div class=\"error\" style=\"color:red;\">${err}</div>`;\n",
              "      throw err;\n",
              "    }\n",
              "\n",
              "    function displayChart(vegaEmbed) {\n",
              "      vegaEmbed(outputDiv, spec, embedOpt)\n",
              "        .catch(err => showError(`Javascript Error: ${err.message}<br>This usually means there's a typo in your chart specification. See the javascript console for the full traceback.`));\n",
              "    }\n",
              "\n",
              "    if(typeof define === \"function\" && define.amd) {\n",
              "      requirejs.config({paths});\n",
              "      require([\"vega-embed\"], displayChart, err => showError(`Error loading script: ${err.message}`));\n",
              "    } else {\n",
              "      maybeLoadScript(\"vega\", \"5\")\n",
              "        .then(() => maybeLoadScript(\"vega-lite\", \"5.17.0\"))\n",
              "        .then(() => maybeLoadScript(\"vega-embed\", \"6\"))\n",
              "        .catch(showError)\n",
              "        .then(() => displayChart(vegaEmbed));\n",
              "    }\n",
              "  })({\"config\": {\"view\": {\"continuousWidth\": 300, \"continuousHeight\": 300}, \"params\": [{\"name\": \"gamma_sel\", \"bind\": {\"input\": \"select\", \"options\": [3], \"labels\": [\"Exact match on first_name (TF col: first_name)\"], \"name\": \"Gamma level:\"}, \"value\": 3}]}, \"hconcat\": [{\"layer\": [{\"mark\": {\"type\": \"point\", \"filled\": true, \"size\": 100, \"stroke\": \"black\", \"strokeWidth\": 1}, \"encoding\": {\"color\": {\"field\": \"log2_bf_tf\", \"scale\": {\"domain\": [-2.5, 2.5], \"scheme\": \"redyellowgreen\"}, \"title\": \"TF adjustment weight\", \"type\": \"quantitative\"}, \"tooltip\": [{\"field\": \"value\", \"title\": \"Value\", \"type\": \"nominal\"}, {\"field\": \"log2_bf\", \"format\": \"+.3\", \"title\": \"Match weight\", \"type\": \"quantitative\"}, {\"field\": \"log2_bf_tf\", \"format\": \"+.3\", \"title\": \"TF adjusted match weight\", \"type\": \"quantitative\"}, {\"field\": \"log2_bf_final\", \"format\": \"+.3\", \"title\": \"Final match weight\", \"type\": \"quantitative\"}], \"x\": {\"axis\": {\"labelAngle\": -60, \"labelFontSize\": 16, \"titleFontSize\": 20}, \"field\": \"value\", \"sort\": {\"field\": \"log2_bf_final\", \"order\": \"ascending\"}, \"title\": \"TF column value\", \"type\": \"nominal\"}, \"y\": {\"axis\": {\"format\": \"+\", \"labelFontSize\": 16, \"titleFontSize\": 18, \"values\": [-5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]}, \"field\": \"log2_bf_final\", \"title\": \"Match weight\", \"type\": \"quantitative\"}}}, {\"mark\": \"rule\", \"encoding\": {\"y\": {\"field\": \"log2_bf\", \"type\": \"quantitative\"}}, \"transform\": [{\"filter\": \"datum.gamma == gamma_sel\"}]}, {\"mark\": {\"type\": \"rule\", \"opacity\": 0.5, \"strokeWidth\": 2}, \"encoding\": {\"color\": {\"field\": \"log2_bf_tf\", \"legend\": null, \"scale\": {\"domain\": [-2.5, 2.5], \"scheme\": \"redyellowgreen\"}, \"title\": \"TF adjustment weight\", \"type\": \"quantitative\"}, \"x\": {\"field\": \"value\", \"sort\": {\"field\": \"log2_bf_final\", \"order\": \"ascending\"}, \"title\": \"TF column value\", \"type\": \"nominal\"}, \"y\": {\"field\": \"log2_bf_final\", \"type\": \"quantitative\"}, \"y2\": {\"type\": \"quantitative\"}}, \"transform\": [{\"filter\": \"datum.gamma == gamma_sel\"}]}], \"data\": {\"name\": \"data\"}, \"height\": 400, \"transform\": [{\"filter\": \"datum.gamma == gamma_sel\"}], \"width\": {\"step\": 20}}, {\"mark\": {\"type\": \"bar\", \"fillOpacity\": 0.8, \"filled\": true, \"stroke\": \"black\", \"strokeWidth\": 1}, \"data\": {\"name\": \"hist\"}, \"encoding\": {\"color\": {\"field\": \"log2_bf_tf\", \"legend\": null, \"scale\": {\"domain\": [-2.5, 2.5], \"scheme\": \"redyellowgreen\"}, \"title\": \"TF adjustment weight\", \"type\": \"quantitative\"}, \"tooltip\": [{\"field\": \"log2_bf_desc\", \"title\": \"Match weight\"}, {\"field\": \"count\", \"title\": \"Number of values\"}], \"x\": {\"axis\": {\"domain\": false, \"gridOpacity\": 0.5, \"labelAlign\": \"center\", \"labelFontSize\": 12, \"labelOpacity\": 0.5, \"labelOverlap\": true, \"ticks\": false, \"title\": \"Count of values\", \"titleFontSize\": 12, \"titleOpacity\": 0.5}, \"field\": \"count\", \"type\": \"quantitative\"}, \"y\": {\"axis\": null, \"bin\": {\"step\": 0.5}, \"field\": \"log2_bf_final\", \"type\": \"quantitative\"}}, \"transform\": [{\"filter\": {\"or\": [\"datum.gamma == gamma_sel\", {\"field\": \"gamma\", \"valid\": false}]}}], \"view\": {\"stroke\": \"transparent\"}, \"width\": 100}], \"datasets\": {\"data\": [{\"value\": \"Oscra\", \"tf\": 0.0012033694344163659, \"label_for_charts\": \"Exact match on first_name\", \"u_probability\": 0.0057935713975033705, \"tf_col\": \"first_name\", \"tf_adjustment_weight\": 1.0, \"log2_bf\": 6.40519430138448, \"gamma\": 3, \"log2_bf_tf\": 2.267373341558131, \"log2_bf_final\": 8.67256764294261, \"most_freq_rank\": 334, \"least_freq_rank\": 0}, {\"value\": \"foSia\", \"tf\": 0.0012033694344163659, \"label_for_charts\": \"Exact match on first_name\", \"u_probability\": 0.0057935713975033705, \"tf_col\": \"first_name\", \"tf_adjustment_weight\": 1.0, \"log2_bf\": 6.40519430138448, \"gamma\": 3, \"log2_bf_tf\": 2.267373341558131, \"log2_bf_final\": 8.67256764294261, \"most_freq_rank\": 333, \"least_freq_rank\": 1}, {\"value\": \"hNoaN\", \"tf\": 0.0012033694344163659, \"label_for_charts\": \"Exact match on first_name\", \"u_probability\": 0.0057935713975033705, \"tf_col\": \"first_name\", \"tf_adjustment_weight\": 1.0, \"log2_bf\": 6.40519430138448, \"gamma\": 3, \"log2_bf_tf\": 2.267373341558131, \"log2_bf_final\": 8.67256764294261, \"most_freq_rank\": 332, \"least_freq_rank\": 2}, {\"value\": \"EvEa\", \"tf\": 0.0012033694344163659, \"label_for_charts\": \"Exact match on first_name\", \"u_probability\": 0.0057935713975033705, \"tf_col\": \"first_name\", \"tf_adjustment_weight\": 1.0, \"log2_bf\": 6.40519430138448, \"gamma\": 3, \"log2_bf_tf\": 2.267373341558131, \"log2_bf_final\": 8.67256764294261, \"most_freq_rank\": 331, \"least_freq_rank\": 3}, {\"value\": \"Imgeen\", \"tf\": 0.0012033694344163659, \"label_for_charts\": \"Exact match on first_name\", \"u_probability\": 0.0057935713975033705, \"tf_col\": \"first_name\", \"tf_adjustment_weight\": 1.0, \"log2_bf\": 6.40519430138448, \"gamma\": 3, \"log2_bf_tf\": 2.267373341558131, \"log2_bf_final\": 8.67256764294261, \"most_freq_rank\": 330, \"least_freq_rank\": 4}, {\"value\": \"geeorGe\", \"tf\": 0.0012033694344163659, \"label_for_charts\": \"Exact match on first_name\", \"u_probability\": 0.0057935713975033705, \"tf_col\": \"first_name\", \"tf_adjustment_weight\": 1.0, \"log2_bf\": 6.40519430138448, \"gamma\": 3, \"log2_bf_tf\": 2.267373341558131, \"log2_bf_final\": 8.67256764294261, \"most_freq_rank\": 329, \"least_freq_rank\": 5}, {\"value\": \"Lydiia\", \"tf\": 0.0012033694344163659, \"label_for_charts\": \"Exact match on first_name\", \"u_probability\": 0.0057935713975033705, \"tf_col\": \"first_name\", \"tf_adjustment_weight\": 1.0, \"log2_bf\": 6.40519430138448, \"gamma\": 3, \"log2_bf_tf\": 2.267373341558131, \"log2_bf_final\": 8.67256764294261, \"most_freq_rank\": 328, \"least_freq_rank\": 6}, {\"value\": \"RoRyn\", \"tf\": 0.0012033694344163659, \"label_for_charts\": \"Exact match on first_name\", \"u_probability\": 0.0057935713975033705, \"tf_col\": \"first_name\", \"tf_adjustment_weight\": 1.0, \"log2_bf\": 6.40519430138448, \"gamma\": 3, \"log2_bf_tf\": 2.267373341558131, \"log2_bf_final\": 8.67256764294261, \"most_freq_rank\": 327, \"least_freq_rank\": 7}, {\"value\": \"Olirev\", \"tf\": 0.0012033694344163659, \"label_for_charts\": \"Exact match on first_name\", \"u_probability\": 0.0057935713975033705, \"tf_col\": \"first_name\", \"tf_adjustment_weight\": 1.0, \"log2_bf\": 6.40519430138448, \"gamma\": 3, \"log2_bf_tf\": 2.267373341558131, \"log2_bf_final\": 8.67256764294261, \"most_freq_rank\": 326, \"least_freq_rank\": 8}, {\"value\": \"Teheo\", \"tf\": 0.0012033694344163659, \"label_for_charts\": \"Exact match on first_name\", \"u_probability\": 0.0057935713975033705, \"tf_col\": \"first_name\", \"tf_adjustment_weight\": 1.0, \"log2_bf\": 6.40519430138448, \"gamma\": 3, \"log2_bf_tf\": 2.267373341558131, \"log2_bf_final\": 8.67256764294261, \"most_freq_rank\": 325, \"least_freq_rank\": 9}, {\"value\": \"Robert\", \"tf\": 0.0036101083032490976, \"label_for_charts\": \"Exact match on first_name\", \"u_probability\": 0.0057935713975033705, \"tf_col\": \"first_name\", \"tf_adjustment_weight\": 1.0, \"log2_bf\": 6.40519430138448, \"gamma\": 3, \"log2_bf_tf\": 0.6824108408369749, \"log2_bf_final\": 7.087605142221455, \"most_freq_rank\": 84, \"least_freq_rank\": 250}, {\"value\": \"Grace\", \"tf\": 0.006016847172081829, \"label_for_charts\": \"Exact match on first_name\", \"u_probability\": 0.0057935713975033705, \"tf_col\": \"first_name\", \"tf_adjustment_weight\": 1.0, \"log2_bf\": 6.40519430138448, \"gamma\": 3, \"log2_bf_tf\": -0.05455475332923131, \"log2_bf_final\": 6.350639548055248, \"most_freq_rank\": 42, \"least_freq_rank\": 292}, {\"value\": \"Theodore\", \"tf\": 0.012033694344163659, \"label_for_charts\": \"Exact match on first_name\", \"u_probability\": 0.0057935713975033705, \"tf_col\": \"first_name\", \"tf_adjustment_weight\": 1.0, \"log2_bf\": 6.40519430138448, \"gamma\": 3, \"log2_bf_tf\": -1.0545547533292312, \"log2_bf_final\": 5.350639548055248, \"most_freq_rank\": 9, \"least_freq_rank\": 325}, {\"value\": \"Elizabeth\", \"tf\": 0.013237063778580024, \"label_for_charts\": \"Exact match on first_name\", \"u_probability\": 0.0057935713975033705, \"tf_col\": \"first_name\", \"tf_adjustment_weight\": 1.0, \"log2_bf\": 6.40519430138448, \"gamma\": 3, \"log2_bf_tf\": -1.1920582770791661, \"log2_bf_final\": 5.213136024305314, \"most_freq_rank\": 8, \"least_freq_rank\": 326}, {\"value\": \"Alfie\", \"tf\": 0.013237063778580024, \"label_for_charts\": \"Exact match on first_name\", \"u_probability\": 0.0057935713975033705, \"tf_col\": \"first_name\", \"tf_adjustment_weight\": 1.0, \"log2_bf\": 6.40519430138448, \"gamma\": 3, \"log2_bf_tf\": -1.1920582770791661, \"log2_bf_final\": 5.213136024305314, \"most_freq_rank\": 7, \"least_freq_rank\": 327}, {\"value\": \"Jessica\", \"tf\": 0.013237063778580024, \"label_for_charts\": \"Exact match on first_name\", \"u_probability\": 0.0057935713975033705, \"tf_col\": \"first_name\", \"tf_adjustment_weight\": 1.0, \"log2_bf\": 6.40519430138448, \"gamma\": 3, \"log2_bf_tf\": -1.1920582770791661, \"log2_bf_final\": 5.213136024305314, \"most_freq_rank\": 6, \"least_freq_rank\": 328}, {\"value\": \"George\", \"tf\": 0.01444043321299639, \"label_for_charts\": \"Exact match on first_name\", \"u_probability\": 0.0057935713975033705, \"tf_col\": \"first_name\", \"tf_adjustment_weight\": 1.0, \"log2_bf\": 6.40519430138448, \"gamma\": 3, \"log2_bf_tf\": -1.317589159163025, \"log2_bf_final\": 5.087605142221455, \"most_freq_rank\": 5, \"least_freq_rank\": 329}, {\"value\": \"James\", \"tf\": 0.015643802647412757, \"label_for_charts\": \"Exact match on first_name\", \"u_probability\": 0.0057935713975033705, \"tf_col\": \"first_name\", \"tf_adjustment_weight\": 1.0, \"log2_bf\": 6.40519430138448, \"gamma\": 3, \"log2_bf_tf\": -1.4330663765829612, \"log2_bf_final\": 4.972127924801518, \"most_freq_rank\": 4, \"least_freq_rank\": 330}, {\"value\": \"Olivia\", \"tf\": 0.01684717208182912, \"label_for_charts\": \"Exact match on first_name\", \"u_probability\": 0.0057935713975033705, \"tf_col\": \"first_name\", \"tf_adjustment_weight\": 1.0, \"log2_bf\": 6.40519430138448, \"gamma\": 3, \"log2_bf_tf\": -1.539981580499473, \"log2_bf_final\": 4.8652127208850064, \"most_freq_rank\": 3, \"least_freq_rank\": 331}, {\"value\": \"Freddie\", \"tf\": 0.018050541516245487, \"label_for_charts\": \"Exact match on first_name\", \"u_probability\": 0.0057935713975033705, \"tf_col\": \"first_name\", \"tf_adjustment_weight\": 1.0, \"log2_bf\": 6.40519430138448, \"gamma\": 3, \"log2_bf_tf\": -1.6395172540503875, \"log2_bf_final\": 4.765677047334092, \"most_freq_rank\": 2, \"least_freq_rank\": 332}, {\"value\": \"Jacob\", \"tf\": 0.019253910950661854, \"label_for_charts\": \"Exact match on first_name\", \"u_probability\": 0.0057935713975033705, \"tf_col\": \"first_name\", \"tf_adjustment_weight\": 1.0, \"log2_bf\": 6.40519430138448, \"gamma\": 3, \"log2_bf_tf\": -1.7326266584418688, \"log2_bf_final\": 4.672567642942611, \"most_freq_rank\": 1, \"least_freq_rank\": 333}, {\"value\": \"Oliver\", \"tf\": 0.03369434416365824, \"label_for_charts\": \"Exact match on first_name\", \"u_probability\": 0.0057935713975033705, \"tf_col\": \"first_name\", \"tf_adjustment_weight\": 1.0, \"log2_bf\": 6.40519430138448, \"gamma\": 3, \"log2_bf_tf\": -2.539981580499473, \"log2_bf_final\": 3.865212720885007, \"most_freq_rank\": 0, \"least_freq_rank\": 334}], \"hist\": [{\"gamma\": 3, \"log2_bf\": 6.40519430138448, \"count\": 1, \"log2_bf_tf\": -2.539981580499473, \"bin_start\": 3.5, \"bin_end\": 4.0, \"log2_bf_final\": 3.75, \"log2_bf_desc\": \"3.5-4.0\"}, {\"gamma\": 3, \"log2_bf\": 6.40519430138448, \"count\": 4, \"log2_bf_tf\": -1.5862979673936726, \"bin_start\": 4.5, \"bin_end\": 5.0, \"log2_bf_final\": 4.75, \"log2_bf_desc\": \"4.5-5.0\"}, {\"gamma\": 3, \"log2_bf\": 6.40519430138448, \"count\": 6, \"log2_bf_tf\": -1.1671455828431643, \"bin_start\": 5.0, \"bin_end\": 5.5, \"log2_bf_final\": 5.25, \"log2_bf_desc\": \"5.0-5.5\"}, {\"gamma\": 3, \"log2_bf\": 6.40519430138448, \"count\": 12, \"log2_bf_tf\": -0.739213716395367, \"bin_start\": 5.5, \"bin_end\": 6.0, \"log2_bf_final\": 5.75, \"log2_bf_desc\": \"5.5-6.0\"}, {\"gamma\": 3, \"log2_bf\": 6.40519430138448, \"count\": 25, \"log2_bf_tf\": -0.15976851566274886, \"bin_start\": 6.0, \"bin_end\": 6.5, \"log2_bf_final\": 6.25, \"log2_bf_desc\": \"6.0-6.5\"}, {\"gamma\": 3, \"log2_bf\": 6.40519430138448, \"count\": 24, \"log2_bf_tf\": 0.2673733415581312, \"bin_start\": 6.5, \"bin_end\": 7.0, \"log2_bf_final\": 6.75, \"log2_bf_desc\": \"6.5-7.0\"}, {\"gamma\": 3, \"log2_bf\": 6.40519430138448, \"count\": 24, \"log2_bf_tf\": 0.6824108408369748, \"bin_start\": 7.0, \"bin_end\": 7.5, \"log2_bf_final\": 7.25, \"log2_bf_desc\": \"7.0-7.5\"}, {\"gamma\": 3, \"log2_bf\": 6.40519430138448, \"count\": 41, \"log2_bf_tf\": 1.2673733415581312, \"bin_start\": 7.5, \"bin_end\": 8.0, \"log2_bf_final\": 7.75, \"log2_bf_desc\": \"7.5-8.0\"}, {\"gamma\": 3, \"log2_bf\": 6.40519430138448, \"count\": 198, \"log2_bf_tf\": 2.267373341558131, \"bin_start\": 8.5, \"bin_end\": 9.0, \"log2_bf_final\": 8.75, \"log2_bf_desc\": \"8.5-9.0\"}]}, \"resolve\": {\"scale\": {\"color\": \"shared\", \"y\": \"shared\"}}, \"spacing\": 10, \"title\": {\"text\": \"Term frequency adjusted match weights\", \"anchor\": \"middle\", \"fontSize\": 16, \"subtitle\": \"For selected values, incl. the lowest and highest frequency\"}, \"$schema\": \"https://vega.github.io/schema/vega-lite/v5.9.3.json\"}, {\"mode\": \"vega-lite\"});\n",
              "</script>"
            ],
            "text/plain": [
              "alt.HConcatChart(...)"
            ]
          },
          "execution_count": 2,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "import splink.comparison_library as cl\n",
        "from splink import DuckDBAPI, Linker, SettingsCreator, block_on, splink_datasets\n",
        "\n",
        "df = splink_datasets.fake_1000\n",
        "\n",
        "settings = SettingsCreator(\n",
        "    link_type=\"dedupe_only\",\n",
        "    comparisons=[\n",
        "        cl.JaroWinklerAtThresholds(\"first_name\", [0.9, 0.7]).configure(\n",
        "            term_frequency_adjustments=True\n",
        "        ),\n",
        "        cl.JaroAtThresholds(\"surname\", [0.9, 0.7]),\n",
        "        cl.DateOfBirthComparison(\n",
        "            \"dob\",\n",
        "            input_is_string=True,\n",
        "            datetime_metrics=[\"year\", \"month\"],\n",
        "            datetime_thresholds=[1, 1],\n",
        "        ),\n",
        "        cl.ExactMatch(\"city\").configure(term_frequency_adjustments=True),\n",
        "        cl.EmailComparison(\"email\"),\n",
        "    ],\n",
        "    blocking_rules_to_generate_predictions=[\n",
        "        block_on(\"first_name\"),\n",
        "        block_on(\"surname\"),\n",
        "    ],\n",
        ")\n",
        "\n",
        "linker = Linker(df, settings, DuckDBAPI())\n",
        "linker.training.estimate_u_using_random_sampling(max_pairs=1e6)\n",
        "\n",
        "blocking_rule_for_training = block_on(\"first_name\", \"surname\")\n",
        "linker.training.estimate_parameters_using_expectation_maximisation(\n",
        "    blocking_rule_for_training\n",
        ")\n",
        "\n",
        "blocking_rule_for_training = block_on(\"dob\")\n",
        "linker.training.estimate_parameters_using_expectation_maximisation(\n",
        "    blocking_rule_for_training\n",
        ")\n",
        "\n",
        "chart = linker.visualisations.tf_adjustment_chart(\n",
        "    \"first_name\", vals_to_include=[\"Robert\", \"Grace\"]\n",
        ")\n",
        "chart"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "base",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.8"
    },
    "orig_nbformat": 4
  },
  "nbformat": 4,
  "nbformat_minor": 2
}
